{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "from scipy import stats\n",
    "import warnings;\n",
    "from pysqldf import SQLDF\n",
    "import pandasql as psql\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "from sklearn.model_selection import KFold\n",
    "import sklearn.ensemble as ske\n",
    "import lightgbm as lgb\n",
    "from pandas.api.types import is_string_dtype\n",
    "from pandas.api.types import is_numeric_dtype\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    6019.000000\n",
       "mean        2.018429\n",
       "std         0.748221\n",
       "min         0.364643\n",
       "25%         1.504077\n",
       "50%         1.893112\n",
       "75%         2.393339\n",
       "max         5.081404\n",
       "Name: Price, dtype: float64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train01 = pd.read_csv(\"C:\\\\Kaggle\\\\Cars\\\\Data\\\\TrnDataForLGB.csv\")\n",
    "test01 = pd.read_csv(\"C:\\\\Kaggle\\\\Cars\\\\Data\\\\TstDataForLGB.csv\")\n",
    "train01['Price'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    6019.000000\n",
       "mean        1.204875\n",
       "std         0.107454\n",
       "min         0.921182\n",
       "25%         1.133462\n",
       "50%         1.188852\n",
       "75%         1.258295\n",
       "max         1.661162\n",
       "Name: Price, dtype: float64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train01['Price'] = np.exp(train01['Price']) - 1\n",
    "train01['Price'] = train01['Price']**0.1\n",
    "train01['Price'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CompName_PORSCHE',\n",
       " 'CompName_SKODA',\n",
       " 'CompName_FORD',\n",
       " 'CompName_MAHINDRA',\n",
       " 'CompNameCarName_Q5',\n",
       " 'CompName_HONDA',\n",
       " 'CompNameCarName_ETIOS',\n",
       " 'CompNameCarName_SCORPIO',\n",
       " 'Fuel_TypeDiesel',\n",
       " 'CompName_BMW',\n",
       " 'CompName_CHEVROLET',\n",
       " 'CompNameCarName_B',\n",
       " 'CompNameCarName_MOBILIO',\n",
       " 'CompName_VOLKSWAGEN',\n",
       " 'CompNameCarName_ELITE',\n",
       " 'CompNameCarName_BOLERO',\n",
       " 'CompName_RENAULT',\n",
       " 'LocationDelhi',\n",
       " 'New_Price',\n",
       " 'Lag_Price2',\n",
       " 'CompNameCarName_E-CLASS',\n",
       " 'CompNameCarName_FIESTA',\n",
       " 'LocationPune',\n",
       " 'CompNameCarName_SSANGYONG',\n",
       " 'CompNameCarName_LAURA',\n",
       " 'LocationKochi',\n",
       " 'Owner_TypeThird',\n",
       " 'CompNameCarName_BRIO',\n",
       " 'CompNameCarName_DUSTER',\n",
       " 'Year',\n",
       " 'CompNameCarName_XYLO',\n",
       " 'Owner_TypeSecond',\n",
       " 'CompNameCarName_KUV',\n",
       " 'CompName_JEEP',\n",
       " 'CompNameCarName_ACCENT',\n",
       " 'CompNameCarName_OPTRA',\n",
       " 'CompNameCarName_DZIRE',\n",
       " 'CompNameCarName_SUNNY',\n",
       " 'CompNameCarName_A-STAR',\n",
       " 'CompName_MINI',\n",
       " 'CompNameCarName_ERTIGA',\n",
       " 'CompName_DATSUN',\n",
       " 'CompNameCarName_CR-V',\n",
       " 'CompNameCarName_CRETA',\n",
       " 'CompNameCarName_FIGO',\n",
       " 'TransmissionAutomatic',\n",
       " 'CompNameCarName_MANZA',\n",
       " 'CompNameCarName_COOPER',\n",
       " 'CompNameCarName_MICRA',\n",
       " 'CompNameCarName_PAJERO',\n",
       " 'CompNameCarName_SWIFT',\n",
       " 'CompNameCarName_Q3',\n",
       " 'LocationJaipur',\n",
       " 'CompNameCarName_SANTA',\n",
       " 'CompNameCarName_BALENO',\n",
       " 'CompNameCarName_CITY',\n",
       " 'Fuel_TypePetrol',\n",
       " 'CompNameCarName_A4',\n",
       " 'LocationCoimbatore',\n",
       " 'CompName_VOLVO',\n",
       " 'CompNameCarName_S',\n",
       " 'CompNameCarName_ROVER',\n",
       " 'CompNameCarName_ENDEAVOUR',\n",
       " 'CompNameCarName_XUV500',\n",
       " 'Fuel_TypeCNG',\n",
       " 'CompNameCarName_AMAZE',\n",
       " 'CompNameCarName_GLE',\n",
       " 'CompNameCarName_FORTUNER',\n",
       " 'CompNameCarName_AVEO',\n",
       " 'CompNameCarName_CLA',\n",
       " 'Seats',\n",
       " 'CompNameCarName_TIAGO',\n",
       " 'CompNameCarName_ZEN',\n",
       " 'LocationMumbai',\n",
       " 'CompNameCarName_INDIGO',\n",
       " 'CompNameCarName_SX4',\n",
       " 'CompName_MITSUBISHI',\n",
       " 'Owner_TypeFourth...Above',\n",
       " 'LocationKolkata',\n",
       " 'Mileage',\n",
       " 'CompNameCarName_SANTRO',\n",
       " 'CompName_AUDI',\n",
       " 'CompNameCarName_M-CLASS',\n",
       " 'CompNameCarName_GL-CLASS',\n",
       " 'CompNameCarName_JAZZ',\n",
       " 'LocationChennai',\n",
       " 'CompNameCarName_X1',\n",
       " 'CompNameCarName_X3',\n",
       " 'CompNameCarName_ECOSPORT',\n",
       " 'LocationHyderabad',\n",
       " 'CompNameCarName_GRAND',\n",
       " 'CompName_JAGUAR',\n",
       " 'Kilometers_Driven',\n",
       " 'TransmissionManual',\n",
       " 'Fuel_TypeElectric',\n",
       " 'CompNameCarName_7',\n",
       " 'CompNameCarName_Q7',\n",
       " 'Lag_Price4',\n",
       " 'LocationAhmedabad',\n",
       " 'CompNameCarName_INNOVA',\n",
       " 'CompNameCarName_3',\n",
       " 'CompNameCarName_OMNI',\n",
       " 'CompNameCarName_RAPID',\n",
       " 'CompNameCarName_IKON',\n",
       " 'CompName_TATA',\n",
       " 'CompNameCarName_TERRANO',\n",
       " 'CompNameCarName_ZEST',\n",
       " 'Owner_TypeFirst',\n",
       " 'CompNameCarName_OCTAVIA',\n",
       " 'CompNameCarName_NANO',\n",
       " 'CompNameCarName_SUPERB',\n",
       " 'CompNameCarName_XF',\n",
       " 'CompNameCarName_ACCORD',\n",
       " 'CompNameCarName_CRUZE',\n",
       " 'CompNameCarName_I10',\n",
       " 'CompName_MERCEDES-BENZ',\n",
       " 'CompNameCarName_800',\n",
       " 'CompName_MARUTI',\n",
       " 'CompNameCarName_COROLLA',\n",
       " 'CompNameCarName_POLO',\n",
       " 'CompNameCarName_AMEO',\n",
       " 'CompNameCarName_JETTA',\n",
       " 'CompNameCarName_COMPASS',\n",
       " 'CompNameCarName_NEW',\n",
       " 'CompName_FIAT',\n",
       " 'CompNameCarName_CAMRY',\n",
       " 'CompNameCarName_LINEA',\n",
       " 'Fuel_TypeLPG',\n",
       " 'CompNameCarName_CELERIO',\n",
       " 'CompNameCarName_ELANTRA',\n",
       " 'CompNameCarName_FABIA',\n",
       " 'CompName_NISSAN',\n",
       " 'CompNameCarName_VERNA',\n",
       " 'CompNameCarName_EON',\n",
       " 'CompNameCarName_CIVIC',\n",
       " 'CompNameCarName_XCENT',\n",
       " 'CompNameCarName_RITZ',\n",
       " 'CompNameCarName_WAGON',\n",
       " 'CompNameCarName_X5',\n",
       " 'CompNameCarName_5',\n",
       " 'CompNameCarName_BEAT',\n",
       " 'CompName_LAND',\n",
       " 'Power',\n",
       " 'CompNameCarName_A6',\n",
       " 'CompNameCarName_VENTO',\n",
       " 'CompNameCarName_EECO',\n",
       " 'CompNameCarName_VITARA',\n",
       " 'CompNameCarName_CIAZ',\n",
       " 'LocationBangalore',\n",
       " 'CompNameCarName_KWID',\n",
       " 'CompNameCarName_ALTO',\n",
       " 'CompName_TOYOTA',\n",
       " 'CompNameCarName_I20',\n",
       " 'CompNameCarName_GLA',\n",
       " 'Engine',\n",
       " 'CompName_HYUNDAI',\n",
       " 'CompNameCarName_INDICA']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Remove_List = [\"id\",\"Price\",\"Name\",\"Lag_Price2_MIN\",\"Lag_Price2_MAX\",\n",
    "               \"Lag_Price3_MIN\",\"Lag_Price3_MAX\",\"Lag_Price\",\"Lag_Price3\",\"Engine_Group\",\n",
    "               \"Power_Group\",\"TrainTestInd\",\"CarCompName\",\"RateChng1\",\"RateChng2\",\"RateChng3\",\n",
    "               \"Lag_Price4_MIN\",\"Lag_Price4_MAX\",\"Lag_Price4_MIN_BY_MAX\"]\n",
    "feature_names = list(set(list(train01.columns)) - set(Remove_List))\n",
    "feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train02 = train01.copy(deep=True)\n",
    "train02.reset_index(drop = True, inplace = True)\n",
    "kf = KFold(n_splits = 5, shuffle = True, random_state = 100)\n",
    "kf.get_n_splits(train02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running CV Iteration Num : 1\n",
      "Training until validation scores don't improve for 400 rounds.\n",
      "[200]\tvalid_0's rmse: 0.0616187\n",
      "[400]\tvalid_0's rmse: 0.0411833\n",
      "[600]\tvalid_0's rmse: 0.0323872\n",
      "[800]\tvalid_0's rmse: 0.0285635\n",
      "[1000]\tvalid_0's rmse: 0.0264913\n",
      "[1200]\tvalid_0's rmse: 0.0251824\n",
      "[1400]\tvalid_0's rmse: 0.0242974\n",
      "[1600]\tvalid_0's rmse: 0.023698\n",
      "[1800]\tvalid_0's rmse: 0.0232132\n",
      "[2000]\tvalid_0's rmse: 0.0228078\n",
      "[2200]\tvalid_0's rmse: 0.0225097\n",
      "[2400]\tvalid_0's rmse: 0.0222466\n",
      "[2600]\tvalid_0's rmse: 0.0219975\n",
      "[2800]\tvalid_0's rmse: 0.0217572\n",
      "[3000]\tvalid_0's rmse: 0.0215397\n",
      "[3200]\tvalid_0's rmse: 0.0213411\n",
      "[3400]\tvalid_0's rmse: 0.0211763\n",
      "[3600]\tvalid_0's rmse: 0.0210183\n",
      "[3800]\tvalid_0's rmse: 0.0208797\n",
      "[4000]\tvalid_0's rmse: 0.0207497\n",
      "[4200]\tvalid_0's rmse: 0.0206242\n",
      "[4400]\tvalid_0's rmse: 0.0205275\n",
      "[4600]\tvalid_0's rmse: 0.0204206\n",
      "[4800]\tvalid_0's rmse: 0.0203353\n",
      "[5000]\tvalid_0's rmse: 0.0202459\n",
      "[5200]\tvalid_0's rmse: 0.0201587\n",
      "[5400]\tvalid_0's rmse: 0.0200846\n",
      "[5600]\tvalid_0's rmse: 0.02001\n",
      "[5800]\tvalid_0's rmse: 0.0199524\n",
      "[6000]\tvalid_0's rmse: 0.0198932\n",
      "[6200]\tvalid_0's rmse: 0.019834\n",
      "[6400]\tvalid_0's rmse: 0.0197788\n",
      "[6600]\tvalid_0's rmse: 0.0197215\n",
      "[6800]\tvalid_0's rmse: 0.0196685\n",
      "[7000]\tvalid_0's rmse: 0.0196241\n",
      "[7200]\tvalid_0's rmse: 0.0195844\n",
      "[7400]\tvalid_0's rmse: 0.0195394\n",
      "[7600]\tvalid_0's rmse: 0.0195028\n",
      "[7800]\tvalid_0's rmse: 0.019462\n",
      "[8000]\tvalid_0's rmse: 0.0194236\n",
      "[8200]\tvalid_0's rmse: 0.0193887\n",
      "[8400]\tvalid_0's rmse: 0.0193518\n",
      "[8600]\tvalid_0's rmse: 0.0193179\n",
      "[8800]\tvalid_0's rmse: 0.0192879\n",
      "[9000]\tvalid_0's rmse: 0.0192635\n",
      "[9200]\tvalid_0's rmse: 0.0192258\n",
      "[9400]\tvalid_0's rmse: 0.0191951\n",
      "[9600]\tvalid_0's rmse: 0.0191662\n",
      "[9800]\tvalid_0's rmse: 0.0191389\n",
      "[10000]\tvalid_0's rmse: 0.0191133\n",
      "[10200]\tvalid_0's rmse: 0.0190891\n",
      "[10400]\tvalid_0's rmse: 0.0190633\n",
      "[10600]\tvalid_0's rmse: 0.0190337\n",
      "[10800]\tvalid_0's rmse: 0.0190143\n",
      "[11000]\tvalid_0's rmse: 0.0189916\n",
      "[11200]\tvalid_0's rmse: 0.0189702\n",
      "[11400]\tvalid_0's rmse: 0.0189526\n",
      "[11600]\tvalid_0's rmse: 0.0189282\n",
      "[11800]\tvalid_0's rmse: 0.0189042\n",
      "[12000]\tvalid_0's rmse: 0.0188922\n",
      "[12200]\tvalid_0's rmse: 0.0188789\n",
      "[12400]\tvalid_0's rmse: 0.0188609\n",
      "[12600]\tvalid_0's rmse: 0.018846\n",
      "[12800]\tvalid_0's rmse: 0.0188293\n",
      "[13000]\tvalid_0's rmse: 0.0188118\n",
      "[13200]\tvalid_0's rmse: 0.0187953\n",
      "[13400]\tvalid_0's rmse: 0.0187761\n",
      "[13600]\tvalid_0's rmse: 0.0187529\n",
      "[13800]\tvalid_0's rmse: 0.0187376\n",
      "[14000]\tvalid_0's rmse: 0.0187246\n",
      "[14200]\tvalid_0's rmse: 0.0187093\n",
      "[14400]\tvalid_0's rmse: 0.0186964\n",
      "[14600]\tvalid_0's rmse: 0.0186746\n",
      "[14800]\tvalid_0's rmse: 0.0186596\n",
      "[15000]\tvalid_0's rmse: 0.0186449\n",
      "[15200]\tvalid_0's rmse: 0.0186354\n",
      "[15400]\tvalid_0's rmse: 0.0186212\n",
      "[15600]\tvalid_0's rmse: 0.0186069\n",
      "[15800]\tvalid_0's rmse: 0.0185967\n",
      "[16000]\tvalid_0's rmse: 0.018587\n",
      "[16200]\tvalid_0's rmse: 0.0185794\n",
      "[16400]\tvalid_0's rmse: 0.018571\n",
      "[16600]\tvalid_0's rmse: 0.0185695\n",
      "[16800]\tvalid_0's rmse: 0.0185645\n",
      "[17000]\tvalid_0's rmse: 0.0185517\n",
      "[17200]\tvalid_0's rmse: 0.0185453\n",
      "[17400]\tvalid_0's rmse: 0.0185453\n",
      "[17600]\tvalid_0's rmse: 0.0185349\n",
      "[17800]\tvalid_0's rmse: 0.0185283\n",
      "[18000]\tvalid_0's rmse: 0.0185202\n",
      "[18200]\tvalid_0's rmse: 0.0185113\n",
      "[18400]\tvalid_0's rmse: 0.0185051\n",
      "[18600]\tvalid_0's rmse: 0.018498\n",
      "[18800]\tvalid_0's rmse: 0.0184921\n",
      "[19000]\tvalid_0's rmse: 0.0184912\n",
      "[19200]\tvalid_0's rmse: 0.0184852\n",
      "[19400]\tvalid_0's rmse: 0.0184756\n",
      "[19600]\tvalid_0's rmse: 0.018475\n",
      "[19800]\tvalid_0's rmse: 0.018472\n",
      "[20000]\tvalid_0's rmse: 0.0184708\n",
      "[20200]\tvalid_0's rmse: 0.0184683\n",
      "[20400]\tvalid_0's rmse: 0.0184593\n",
      "[20600]\tvalid_0's rmse: 0.0184527\n",
      "[20800]\tvalid_0's rmse: 0.0184506\n",
      "[21000]\tvalid_0's rmse: 0.0184455\n",
      "[21200]\tvalid_0's rmse: 0.0184442\n",
      "[21400]\tvalid_0's rmse: 0.0184458\n",
      "[21600]\tvalid_0's rmse: 0.0184424\n",
      "[21800]\tvalid_0's rmse: 0.0184397\n",
      "[22000]\tvalid_0's rmse: 0.0184391\n",
      "[22200]\tvalid_0's rmse: 0.0184365\n",
      "[22400]\tvalid_0's rmse: 0.0184354\n",
      "[22600]\tvalid_0's rmse: 0.0184331\n",
      "[22800]\tvalid_0's rmse: 0.0184279\n",
      "[23000]\tvalid_0's rmse: 0.0184245\n",
      "[23200]\tvalid_0's rmse: 0.0184224\n",
      "[23400]\tvalid_0's rmse: 0.0184228\n",
      "[23600]\tvalid_0's rmse: 0.018415\n",
      "[23800]\tvalid_0's rmse: 0.0184139\n",
      "[24000]\tvalid_0's rmse: 0.0184118\n",
      "[24200]\tvalid_0's rmse: 0.0184057\n",
      "[24400]\tvalid_0's rmse: 0.018408\n",
      "[24600]\tvalid_0's rmse: 0.018407\n",
      "[24800]\tvalid_0's rmse: 0.0184051\n",
      "[25000]\tvalid_0's rmse: 0.0184047\n",
      "[25200]\tvalid_0's rmse: 0.0184055\n",
      "[25400]\tvalid_0's rmse: 0.0184084\n",
      "Early stopping, best iteration is:\n",
      "[25024]\tvalid_0's rmse: 0.018404\n",
      "Test RMSE :  0.12372007170991565\n",
      "Running CV Iteration Num : 2\n",
      "Training until validation scores don't improve for 400 rounds.\n",
      "[200]\tvalid_0's rmse: 0.0610417\n",
      "[400]\tvalid_0's rmse: 0.040821\n",
      "[600]\tvalid_0's rmse: 0.0323183\n",
      "[800]\tvalid_0's rmse: 0.0287269\n",
      "[1000]\tvalid_0's rmse: 0.0267997\n",
      "[1200]\tvalid_0's rmse: 0.0255573\n",
      "[1400]\tvalid_0's rmse: 0.0247244\n",
      "[1600]\tvalid_0's rmse: 0.0241537\n",
      "[1800]\tvalid_0's rmse: 0.0237413\n",
      "[2000]\tvalid_0's rmse: 0.0234004\n",
      "[2200]\tvalid_0's rmse: 0.0231197\n",
      "[2400]\tvalid_0's rmse: 0.0228787\n",
      "[2600]\tvalid_0's rmse: 0.022682\n",
      "[2800]\tvalid_0's rmse: 0.0224747\n",
      "[3000]\tvalid_0's rmse: 0.0223033\n",
      "[3200]\tvalid_0's rmse: 0.0221657\n",
      "[3400]\tvalid_0's rmse: 0.0220445\n",
      "[3600]\tvalid_0's rmse: 0.0219406\n",
      "[3800]\tvalid_0's rmse: 0.0218294\n",
      "[4000]\tvalid_0's rmse: 0.0217149\n",
      "[4200]\tvalid_0's rmse: 0.0216189\n",
      "[4400]\tvalid_0's rmse: 0.0215362\n",
      "[4600]\tvalid_0's rmse: 0.0214522\n",
      "[4800]\tvalid_0's rmse: 0.0213664\n",
      "[5000]\tvalid_0's rmse: 0.0212849\n",
      "[5200]\tvalid_0's rmse: 0.0212202\n",
      "[5400]\tvalid_0's rmse: 0.0211559\n",
      "[5600]\tvalid_0's rmse: 0.0210863\n",
      "[5800]\tvalid_0's rmse: 0.0210274\n",
      "[6000]\tvalid_0's rmse: 0.0209491\n",
      "[6200]\tvalid_0's rmse: 0.0208944\n",
      "[6400]\tvalid_0's rmse: 0.0208355\n",
      "[6600]\tvalid_0's rmse: 0.0207818\n",
      "[6800]\tvalid_0's rmse: 0.020727\n",
      "[7000]\tvalid_0's rmse: 0.0206749\n",
      "[7200]\tvalid_0's rmse: 0.0206273\n",
      "[7400]\tvalid_0's rmse: 0.0205899\n",
      "[7600]\tvalid_0's rmse: 0.020546\n",
      "[7800]\tvalid_0's rmse: 0.0205065\n",
      "[8000]\tvalid_0's rmse: 0.0204672\n",
      "[8200]\tvalid_0's rmse: 0.0204366\n",
      "[8400]\tvalid_0's rmse: 0.0203946\n",
      "[8600]\tvalid_0's rmse: 0.020367\n",
      "[8800]\tvalid_0's rmse: 0.0203251\n",
      "[9000]\tvalid_0's rmse: 0.0203017\n",
      "[9200]\tvalid_0's rmse: 0.0202655\n",
      "[9400]\tvalid_0's rmse: 0.0202368\n",
      "[9600]\tvalid_0's rmse: 0.0202114\n",
      "[9800]\tvalid_0's rmse: 0.0201818\n",
      "[10000]\tvalid_0's rmse: 0.0201543\n",
      "[10200]\tvalid_0's rmse: 0.0201207\n",
      "[10400]\tvalid_0's rmse: 0.0200925\n",
      "[10600]\tvalid_0's rmse: 0.0200736\n",
      "[10800]\tvalid_0's rmse: 0.0200477\n",
      "[11000]\tvalid_0's rmse: 0.0200227\n",
      "[11200]\tvalid_0's rmse: 0.0199965\n",
      "[11400]\tvalid_0's rmse: 0.0199827\n",
      "[11600]\tvalid_0's rmse: 0.0199701\n",
      "[11800]\tvalid_0's rmse: 0.0199552\n",
      "[12000]\tvalid_0's rmse: 0.0199388\n",
      "[12200]\tvalid_0's rmse: 0.0199208\n",
      "[12400]\tvalid_0's rmse: 0.0198994\n",
      "[12600]\tvalid_0's rmse: 0.0198796\n",
      "[12800]\tvalid_0's rmse: 0.0198643\n",
      "[13000]\tvalid_0's rmse: 0.0198476\n",
      "[13200]\tvalid_0's rmse: 0.019828\n",
      "[13400]\tvalid_0's rmse: 0.0198097\n",
      "[13600]\tvalid_0's rmse: 0.0197961\n",
      "[13800]\tvalid_0's rmse: 0.0197744\n",
      "[14000]\tvalid_0's rmse: 0.0197611\n",
      "[14200]\tvalid_0's rmse: 0.0197532\n",
      "[14400]\tvalid_0's rmse: 0.0197411\n",
      "[14600]\tvalid_0's rmse: 0.0197257\n",
      "[14800]\tvalid_0's rmse: 0.0197208\n",
      "[15000]\tvalid_0's rmse: 0.0197108\n",
      "[15200]\tvalid_0's rmse: 0.0197019\n",
      "[15400]\tvalid_0's rmse: 0.0196917\n",
      "[15600]\tvalid_0's rmse: 0.0196842\n",
      "[15800]\tvalid_0's rmse: 0.0196782\n",
      "[16000]\tvalid_0's rmse: 0.0196616\n",
      "[16200]\tvalid_0's rmse: 0.0196501\n",
      "[16400]\tvalid_0's rmse: 0.0196372\n",
      "[16600]\tvalid_0's rmse: 0.0196357\n",
      "[16800]\tvalid_0's rmse: 0.0196289\n",
      "[17000]\tvalid_0's rmse: 0.0196196\n",
      "[17200]\tvalid_0's rmse: 0.0196082\n",
      "[17400]\tvalid_0's rmse: 0.0196064\n",
      "[17600]\tvalid_0's rmse: 0.0196023\n",
      "[17800]\tvalid_0's rmse: 0.0195967\n",
      "[18000]\tvalid_0's rmse: 0.0195935\n",
      "[18200]\tvalid_0's rmse: 0.0195864\n",
      "[18400]\tvalid_0's rmse: 0.01958\n",
      "[18600]\tvalid_0's rmse: 0.0195765\n",
      "[18800]\tvalid_0's rmse: 0.0195725\n",
      "[19000]\tvalid_0's rmse: 0.0195684\n",
      "[19200]\tvalid_0's rmse: 0.0195673\n",
      "[19400]\tvalid_0's rmse: 0.0195625\n",
      "[19600]\tvalid_0's rmse: 0.019557\n",
      "[19800]\tvalid_0's rmse: 0.0195528\n",
      "[20000]\tvalid_0's rmse: 0.0195444\n",
      "[20200]\tvalid_0's rmse: 0.0195366\n",
      "[20400]\tvalid_0's rmse: 0.0195347\n",
      "[20600]\tvalid_0's rmse: 0.0195297\n",
      "[20800]\tvalid_0's rmse: 0.0195245\n",
      "[21000]\tvalid_0's rmse: 0.0195151\n",
      "[21200]\tvalid_0's rmse: 0.01951\n",
      "[21400]\tvalid_0's rmse: 0.0195056\n",
      "[21600]\tvalid_0's rmse: 0.0195004\n",
      "[21800]\tvalid_0's rmse: 0.0194953\n",
      "[22000]\tvalid_0's rmse: 0.0194925\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[22200]\tvalid_0's rmse: 0.0194894\n",
      "[22400]\tvalid_0's rmse: 0.0194861\n",
      "[22600]\tvalid_0's rmse: 0.0194839\n",
      "[22800]\tvalid_0's rmse: 0.0194805\n",
      "[23000]\tvalid_0's rmse: 0.0194768\n",
      "[23200]\tvalid_0's rmse: 0.0194758\n",
      "[23400]\tvalid_0's rmse: 0.019471\n",
      "[23600]\tvalid_0's rmse: 0.0194693\n",
      "[23800]\tvalid_0's rmse: 0.0194685\n",
      "[24000]\tvalid_0's rmse: 0.0194715\n",
      "Early stopping, best iteration is:\n",
      "[23655]\tvalid_0's rmse: 0.019467\n",
      "Test RMSE :  0.13257360774070212\n",
      "Running CV Iteration Num : 3\n",
      "Training until validation scores don't improve for 400 rounds.\n",
      "[200]\tvalid_0's rmse: 0.060507\n",
      "[400]\tvalid_0's rmse: 0.041557\n",
      "[600]\tvalid_0's rmse: 0.0344496\n",
      "[800]\tvalid_0's rmse: 0.0316482\n",
      "[1000]\tvalid_0's rmse: 0.0302907\n",
      "[1200]\tvalid_0's rmse: 0.0294192\n",
      "[1400]\tvalid_0's rmse: 0.0288676\n",
      "[1600]\tvalid_0's rmse: 0.028493\n",
      "[1800]\tvalid_0's rmse: 0.0282093\n",
      "[2000]\tvalid_0's rmse: 0.0279861\n",
      "[2200]\tvalid_0's rmse: 0.0277915\n",
      "[2400]\tvalid_0's rmse: 0.027626\n",
      "[2600]\tvalid_0's rmse: 0.027472\n",
      "[2800]\tvalid_0's rmse: 0.0273377\n",
      "[3000]\tvalid_0's rmse: 0.0272278\n",
      "[3200]\tvalid_0's rmse: 0.0271105\n",
      "[3400]\tvalid_0's rmse: 0.0270189\n",
      "[3600]\tvalid_0's rmse: 0.0269236\n",
      "[3800]\tvalid_0's rmse: 0.0268423\n",
      "[4000]\tvalid_0's rmse: 0.0267594\n",
      "[4200]\tvalid_0's rmse: 0.026687\n",
      "[4400]\tvalid_0's rmse: 0.0266189\n",
      "[4600]\tvalid_0's rmse: 0.0265629\n",
      "[4800]\tvalid_0's rmse: 0.0265045\n",
      "[5000]\tvalid_0's rmse: 0.0264627\n",
      "[5200]\tvalid_0's rmse: 0.0264074\n",
      "[5400]\tvalid_0's rmse: 0.0263498\n",
      "[5600]\tvalid_0's rmse: 0.0262967\n",
      "[5800]\tvalid_0's rmse: 0.0262667\n",
      "[6000]\tvalid_0's rmse: 0.0262255\n",
      "[6200]\tvalid_0's rmse: 0.0261846\n",
      "[6400]\tvalid_0's rmse: 0.026135\n",
      "[6600]\tvalid_0's rmse: 0.0260804\n",
      "[6800]\tvalid_0's rmse: 0.0260383\n",
      "[7000]\tvalid_0's rmse: 0.0259976\n",
      "[7200]\tvalid_0's rmse: 0.02597\n",
      "[7400]\tvalid_0's rmse: 0.0259298\n",
      "[7600]\tvalid_0's rmse: 0.0258939\n",
      "[7800]\tvalid_0's rmse: 0.0258643\n",
      "[8000]\tvalid_0's rmse: 0.0258375\n",
      "[8200]\tvalid_0's rmse: 0.0258075\n",
      "[8400]\tvalid_0's rmse: 0.0257927\n",
      "[8600]\tvalid_0's rmse: 0.0257651\n",
      "[8800]\tvalid_0's rmse: 0.0257404\n",
      "[9000]\tvalid_0's rmse: 0.025724\n",
      "[9200]\tvalid_0's rmse: 0.0257084\n",
      "[9400]\tvalid_0's rmse: 0.0256861\n",
      "[9600]\tvalid_0's rmse: 0.0256636\n",
      "[9800]\tvalid_0's rmse: 0.0256525\n",
      "[10000]\tvalid_0's rmse: 0.0256268\n",
      "[10200]\tvalid_0's rmse: 0.0256031\n",
      "[10400]\tvalid_0's rmse: 0.0255812\n",
      "[10600]\tvalid_0's rmse: 0.025568\n",
      "[10800]\tvalid_0's rmse: 0.0255525\n",
      "[11000]\tvalid_0's rmse: 0.0255283\n",
      "[11200]\tvalid_0's rmse: 0.0255215\n",
      "[11400]\tvalid_0's rmse: 0.0255049\n",
      "[11600]\tvalid_0's rmse: 0.0254868\n",
      "[11800]\tvalid_0's rmse: 0.0254765\n",
      "[12000]\tvalid_0's rmse: 0.0254572\n",
      "[12200]\tvalid_0's rmse: 0.0254408\n",
      "[12400]\tvalid_0's rmse: 0.0254305\n",
      "[12600]\tvalid_0's rmse: 0.0254151\n",
      "[12800]\tvalid_0's rmse: 0.0254143\n",
      "[13000]\tvalid_0's rmse: 0.0254023\n",
      "[13200]\tvalid_0's rmse: 0.0253922\n",
      "[13400]\tvalid_0's rmse: 0.0253807\n",
      "[13600]\tvalid_0's rmse: 0.0253668\n",
      "[13800]\tvalid_0's rmse: 0.0253559\n",
      "[14000]\tvalid_0's rmse: 0.0253521\n",
      "[14200]\tvalid_0's rmse: 0.0253377\n",
      "[14400]\tvalid_0's rmse: 0.025324\n",
      "[14600]\tvalid_0's rmse: 0.0253125\n",
      "[14800]\tvalid_0's rmse: 0.0253054\n",
      "[15000]\tvalid_0's rmse: 0.0253007\n",
      "[15200]\tvalid_0's rmse: 0.0252938\n",
      "[15400]\tvalid_0's rmse: 0.025293\n",
      "[15600]\tvalid_0's rmse: 0.0252877\n",
      "[15800]\tvalid_0's rmse: 0.0252833\n",
      "[16000]\tvalid_0's rmse: 0.025278\n",
      "[16200]\tvalid_0's rmse: 0.0252681\n",
      "[16400]\tvalid_0's rmse: 0.0252556\n",
      "[16600]\tvalid_0's rmse: 0.0252502\n",
      "[16800]\tvalid_0's rmse: 0.0252489\n",
      "[17000]\tvalid_0's rmse: 0.0252415\n",
      "[17200]\tvalid_0's rmse: 0.0252338\n",
      "[17400]\tvalid_0's rmse: 0.0252273\n",
      "[17600]\tvalid_0's rmse: 0.0252266\n",
      "[17800]\tvalid_0's rmse: 0.0252263\n",
      "[18000]\tvalid_0's rmse: 0.0252252\n",
      "[18200]\tvalid_0's rmse: 0.0252252\n",
      "[18400]\tvalid_0's rmse: 0.0252275\n",
      "Early stopping, best iteration is:\n",
      "[18076]\tvalid_0's rmse: 0.025222\n",
      "Test RMSE :  0.1735204637755316\n",
      "Running CV Iteration Num : 4\n",
      "Training until validation scores don't improve for 400 rounds.\n",
      "[200]\tvalid_0's rmse: 0.0591376\n",
      "[400]\tvalid_0's rmse: 0.0401976\n",
      "[600]\tvalid_0's rmse: 0.0323337\n",
      "[800]\tvalid_0's rmse: 0.0290072\n",
      "[1000]\tvalid_0's rmse: 0.0271742\n",
      "[1200]\tvalid_0's rmse: 0.0260052\n",
      "[1400]\tvalid_0's rmse: 0.025227\n",
      "[1600]\tvalid_0's rmse: 0.0246748\n",
      "[1800]\tvalid_0's rmse: 0.0242717\n",
      "[2000]\tvalid_0's rmse: 0.0239397\n",
      "[2200]\tvalid_0's rmse: 0.0236735\n",
      "[2400]\tvalid_0's rmse: 0.0234494\n",
      "[2600]\tvalid_0's rmse: 0.0232468\n",
      "[2800]\tvalid_0's rmse: 0.0230571\n",
      "[3000]\tvalid_0's rmse: 0.0229134\n",
      "[3200]\tvalid_0's rmse: 0.0227519\n",
      "[3400]\tvalid_0's rmse: 0.0226188\n",
      "[3600]\tvalid_0's rmse: 0.022505\n",
      "[3800]\tvalid_0's rmse: 0.0223951\n",
      "[4000]\tvalid_0's rmse: 0.0222968\n",
      "[4200]\tvalid_0's rmse: 0.0222068\n",
      "[4400]\tvalid_0's rmse: 0.0221289\n",
      "[4600]\tvalid_0's rmse: 0.0220478\n",
      "[4800]\tvalid_0's rmse: 0.0219644\n",
      "[5000]\tvalid_0's rmse: 0.0218911\n",
      "[5200]\tvalid_0's rmse: 0.0218223\n",
      "[5400]\tvalid_0's rmse: 0.0217527\n",
      "[5600]\tvalid_0's rmse: 0.0216891\n",
      "[5800]\tvalid_0's rmse: 0.0216322\n",
      "[6000]\tvalid_0's rmse: 0.021576\n",
      "[6200]\tvalid_0's rmse: 0.0215166\n",
      "[6400]\tvalid_0's rmse: 0.0214569\n",
      "[6600]\tvalid_0's rmse: 0.0214051\n",
      "[6800]\tvalid_0's rmse: 0.0213459\n",
      "[7000]\tvalid_0's rmse: 0.0212991\n",
      "[7200]\tvalid_0's rmse: 0.0212557\n",
      "[7400]\tvalid_0's rmse: 0.0212138\n",
      "[7600]\tvalid_0's rmse: 0.0211673\n",
      "[7800]\tvalid_0's rmse: 0.0211284\n",
      "[8000]\tvalid_0's rmse: 0.0210869\n",
      "[8200]\tvalid_0's rmse: 0.0210543\n",
      "[8400]\tvalid_0's rmse: 0.0210115\n",
      "[8600]\tvalid_0's rmse: 0.0209825\n",
      "[8800]\tvalid_0's rmse: 0.0209501\n",
      "[9000]\tvalid_0's rmse: 0.0209166\n",
      "[9200]\tvalid_0's rmse: 0.0208861\n",
      "[9400]\tvalid_0's rmse: 0.0208577\n",
      "[9600]\tvalid_0's rmse: 0.0208288\n",
      "[9800]\tvalid_0's rmse: 0.0207989\n",
      "[10000]\tvalid_0's rmse: 0.0207772\n",
      "[10200]\tvalid_0's rmse: 0.0207461\n",
      "[10400]\tvalid_0's rmse: 0.0207198\n",
      "[10600]\tvalid_0's rmse: 0.0206958\n",
      "[10800]\tvalid_0's rmse: 0.0206703\n",
      "[11000]\tvalid_0's rmse: 0.0206503\n",
      "[11200]\tvalid_0's rmse: 0.0206299\n",
      "[11400]\tvalid_0's rmse: 0.0206155\n",
      "[11600]\tvalid_0's rmse: 0.020592\n",
      "[11800]\tvalid_0's rmse: 0.0205732\n",
      "[12000]\tvalid_0's rmse: 0.0205495\n",
      "[12200]\tvalid_0's rmse: 0.0205361\n",
      "[12400]\tvalid_0's rmse: 0.0205087\n",
      "[12600]\tvalid_0's rmse: 0.0204826\n",
      "[12800]\tvalid_0's rmse: 0.0204621\n",
      "[13000]\tvalid_0's rmse: 0.0204429\n",
      "[13200]\tvalid_0's rmse: 0.0204219\n",
      "[13400]\tvalid_0's rmse: 0.0204053\n",
      "[13600]\tvalid_0's rmse: 0.0203873\n",
      "[13800]\tvalid_0's rmse: 0.020376\n",
      "[14000]\tvalid_0's rmse: 0.0203615\n",
      "[14200]\tvalid_0's rmse: 0.0203512\n",
      "[14400]\tvalid_0's rmse: 0.0203437\n",
      "[14600]\tvalid_0's rmse: 0.0203358\n",
      "[14800]\tvalid_0's rmse: 0.0203185\n",
      "[15000]\tvalid_0's rmse: 0.0203058\n",
      "[15200]\tvalid_0's rmse: 0.0202917\n",
      "[15400]\tvalid_0's rmse: 0.0202785\n",
      "[15600]\tvalid_0's rmse: 0.0202673\n",
      "[15800]\tvalid_0's rmse: 0.0202542\n",
      "[16000]\tvalid_0's rmse: 0.0202376\n",
      "[16200]\tvalid_0's rmse: 0.020226\n",
      "[16400]\tvalid_0's rmse: 0.0202181\n",
      "[16600]\tvalid_0's rmse: 0.0202082\n",
      "[16800]\tvalid_0's rmse: 0.0201957\n",
      "[17000]\tvalid_0's rmse: 0.0201866\n",
      "[17200]\tvalid_0's rmse: 0.0201761\n",
      "[17400]\tvalid_0's rmse: 0.020168\n",
      "[17600]\tvalid_0's rmse: 0.0201608\n",
      "[17800]\tvalid_0's rmse: 0.0201526\n",
      "[18000]\tvalid_0's rmse: 0.0201449\n",
      "[18200]\tvalid_0's rmse: 0.0201344\n",
      "[18400]\tvalid_0's rmse: 0.0201294\n",
      "[18600]\tvalid_0's rmse: 0.0201242\n",
      "[18800]\tvalid_0's rmse: 0.0201148\n",
      "[19000]\tvalid_0's rmse: 0.0201081\n",
      "[19200]\tvalid_0's rmse: 0.0201051\n",
      "[19400]\tvalid_0's rmse: 0.0200979\n",
      "[19600]\tvalid_0's rmse: 0.0200876\n",
      "[19800]\tvalid_0's rmse: 0.0200841\n",
      "[20000]\tvalid_0's rmse: 0.0200761\n",
      "[20200]\tvalid_0's rmse: 0.0200665\n",
      "[20400]\tvalid_0's rmse: 0.0200599\n",
      "[20600]\tvalid_0's rmse: 0.020052\n",
      "[20800]\tvalid_0's rmse: 0.0200476\n",
      "[21000]\tvalid_0's rmse: 0.0200416\n",
      "[21200]\tvalid_0's rmse: 0.0200322\n",
      "[21400]\tvalid_0's rmse: 0.0200257\n",
      "[21600]\tvalid_0's rmse: 0.0200194\n",
      "[21800]\tvalid_0's rmse: 0.0200131\n",
      "[22000]\tvalid_0's rmse: 0.0200085\n",
      "[22200]\tvalid_0's rmse: 0.0200059\n",
      "[22400]\tvalid_0's rmse: 0.0199958\n",
      "[22600]\tvalid_0's rmse: 0.019991\n",
      "[22800]\tvalid_0's rmse: 0.0199862\n",
      "[23000]\tvalid_0's rmse: 0.0199831\n",
      "[23200]\tvalid_0's rmse: 0.0199782\n",
      "[23400]\tvalid_0's rmse: 0.0199728\n",
      "[23600]\tvalid_0's rmse: 0.0199696\n",
      "[23800]\tvalid_0's rmse: 0.0199664\n",
      "[24000]\tvalid_0's rmse: 0.0199654\n",
      "[24200]\tvalid_0's rmse: 0.0199649\n",
      "[24400]\tvalid_0's rmse: 0.0199612\n",
      "[24600]\tvalid_0's rmse: 0.0199583\n",
      "[24800]\tvalid_0's rmse: 0.0199548\n",
      "[25000]\tvalid_0's rmse: 0.0199532\n",
      "[25200]\tvalid_0's rmse: 0.0199487\n",
      "[25400]\tvalid_0's rmse: 0.0199474\n",
      "[25600]\tvalid_0's rmse: 0.019944\n",
      "[25800]\tvalid_0's rmse: 0.0199418\n",
      "[26000]\tvalid_0's rmse: 0.0199374\n",
      "[26200]\tvalid_0's rmse: 0.019936\n",
      "[26400]\tvalid_0's rmse: 0.0199361\n",
      "[26600]\tvalid_0's rmse: 0.0199328\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26800]\tvalid_0's rmse: 0.0199321\n",
      "[27000]\tvalid_0's rmse: 0.0199293\n",
      "[27200]\tvalid_0's rmse: 0.0199253\n",
      "[27400]\tvalid_0's rmse: 0.0199229\n",
      "[27600]\tvalid_0's rmse: 0.0199213\n",
      "[27800]\tvalid_0's rmse: 0.019917\n",
      "[28000]\tvalid_0's rmse: 0.0199132\n",
      "[28200]\tvalid_0's rmse: 0.0199104\n",
      "[28400]\tvalid_0's rmse: 0.0199055\n",
      "[28600]\tvalid_0's rmse: 0.0198989\n",
      "[28800]\tvalid_0's rmse: 0.0198953\n",
      "[29000]\tvalid_0's rmse: 0.0198912\n",
      "[29200]\tvalid_0's rmse: 0.0198902\n",
      "[29400]\tvalid_0's rmse: 0.019888\n",
      "[29600]\tvalid_0's rmse: 0.0198885\n",
      "[29800]\tvalid_0's rmse: 0.019887\n",
      "[30000]\tvalid_0's rmse: 0.0198868\n",
      "[30200]\tvalid_0's rmse: 0.0198892\n",
      "Early stopping, best iteration is:\n",
      "[29902]\tvalid_0's rmse: 0.0198855\n",
      "Test RMSE :  0.1373762801199711\n",
      "Running CV Iteration Num : 5\n",
      "Training until validation scores don't improve for 400 rounds.\n",
      "[200]\tvalid_0's rmse: 0.0622608\n",
      "[400]\tvalid_0's rmse: 0.0420566\n",
      "[600]\tvalid_0's rmse: 0.0335168\n",
      "[800]\tvalid_0's rmse: 0.029921\n",
      "[1000]\tvalid_0's rmse: 0.0279789\n",
      "[1200]\tvalid_0's rmse: 0.0267348\n",
      "[1400]\tvalid_0's rmse: 0.0258987\n",
      "[1600]\tvalid_0's rmse: 0.0253409\n",
      "[1800]\tvalid_0's rmse: 0.0248901\n",
      "[2000]\tvalid_0's rmse: 0.0244949\n",
      "[2200]\tvalid_0's rmse: 0.0241862\n",
      "[2400]\tvalid_0's rmse: 0.0239187\n",
      "[2600]\tvalid_0's rmse: 0.0236858\n",
      "[2800]\tvalid_0's rmse: 0.0234616\n",
      "[3000]\tvalid_0's rmse: 0.023277\n",
      "[3200]\tvalid_0's rmse: 0.0231065\n",
      "[3400]\tvalid_0's rmse: 0.0229505\n",
      "[3600]\tvalid_0's rmse: 0.0228121\n",
      "[3800]\tvalid_0's rmse: 0.0226873\n",
      "[4000]\tvalid_0's rmse: 0.0225578\n",
      "[4200]\tvalid_0's rmse: 0.0224342\n",
      "[4400]\tvalid_0's rmse: 0.0223071\n",
      "[4600]\tvalid_0's rmse: 0.0221996\n",
      "[4800]\tvalid_0's rmse: 0.0221041\n",
      "[5000]\tvalid_0's rmse: 0.0220027\n",
      "[5200]\tvalid_0's rmse: 0.0218975\n",
      "[5400]\tvalid_0's rmse: 0.0218001\n",
      "[5600]\tvalid_0's rmse: 0.0217148\n",
      "[5800]\tvalid_0's rmse: 0.0216332\n",
      "[6000]\tvalid_0's rmse: 0.0215499\n",
      "[6200]\tvalid_0's rmse: 0.0214774\n",
      "[6400]\tvalid_0's rmse: 0.0214011\n",
      "[6600]\tvalid_0's rmse: 0.0213258\n",
      "[6800]\tvalid_0's rmse: 0.0212499\n",
      "[7000]\tvalid_0's rmse: 0.0211865\n",
      "[7200]\tvalid_0's rmse: 0.0211154\n",
      "[7400]\tvalid_0's rmse: 0.0210497\n",
      "[7600]\tvalid_0's rmse: 0.0209889\n",
      "[7800]\tvalid_0's rmse: 0.0209264\n",
      "[8000]\tvalid_0's rmse: 0.0208656\n",
      "[8200]\tvalid_0's rmse: 0.0208071\n",
      "[8400]\tvalid_0's rmse: 0.02075\n",
      "[8600]\tvalid_0's rmse: 0.020685\n",
      "[8800]\tvalid_0's rmse: 0.0206343\n",
      "[9000]\tvalid_0's rmse: 0.0205843\n",
      "[9200]\tvalid_0's rmse: 0.0205458\n",
      "[9400]\tvalid_0's rmse: 0.0204998\n",
      "[9600]\tvalid_0's rmse: 0.0204481\n",
      "[9800]\tvalid_0's rmse: 0.0204004\n",
      "[10000]\tvalid_0's rmse: 0.0203569\n",
      "[10200]\tvalid_0's rmse: 0.0203078\n",
      "[10400]\tvalid_0's rmse: 0.0202743\n",
      "[10600]\tvalid_0's rmse: 0.0202385\n",
      "[10800]\tvalid_0's rmse: 0.0201949\n",
      "[11000]\tvalid_0's rmse: 0.0201579\n",
      "[11200]\tvalid_0's rmse: 0.0201152\n",
      "[11400]\tvalid_0's rmse: 0.0200751\n",
      "[11600]\tvalid_0's rmse: 0.0200418\n",
      "[11800]\tvalid_0's rmse: 0.0200182\n",
      "[12000]\tvalid_0's rmse: 0.0199871\n",
      "[12200]\tvalid_0's rmse: 0.0199573\n",
      "[12400]\tvalid_0's rmse: 0.019922\n",
      "[12600]\tvalid_0's rmse: 0.0198868\n",
      "[12800]\tvalid_0's rmse: 0.0198673\n",
      "[13000]\tvalid_0's rmse: 0.0198374\n",
      "[13200]\tvalid_0's rmse: 0.0198087\n",
      "[13400]\tvalid_0's rmse: 0.01979\n",
      "[13600]\tvalid_0's rmse: 0.0197631\n",
      "[13800]\tvalid_0's rmse: 0.0197415\n",
      "[14000]\tvalid_0's rmse: 0.0197166\n",
      "[14200]\tvalid_0's rmse: 0.0196905\n",
      "[14400]\tvalid_0's rmse: 0.0196651\n",
      "[14600]\tvalid_0's rmse: 0.0196429\n",
      "[14800]\tvalid_0's rmse: 0.0196204\n",
      "[15000]\tvalid_0's rmse: 0.0196001\n",
      "[15200]\tvalid_0's rmse: 0.0195766\n",
      "[15400]\tvalid_0's rmse: 0.0195589\n",
      "[15600]\tvalid_0's rmse: 0.01954\n",
      "[15800]\tvalid_0's rmse: 0.0195213\n",
      "[16000]\tvalid_0's rmse: 0.0195014\n",
      "[16200]\tvalid_0's rmse: 0.0194852\n",
      "[16400]\tvalid_0's rmse: 0.01947\n",
      "[16600]\tvalid_0's rmse: 0.0194477\n",
      "[16800]\tvalid_0's rmse: 0.019429\n",
      "[17000]\tvalid_0's rmse: 0.0194202\n",
      "[17200]\tvalid_0's rmse: 0.0194025\n",
      "[17400]\tvalid_0's rmse: 0.0193901\n",
      "[17600]\tvalid_0's rmse: 0.0193684\n",
      "[17800]\tvalid_0's rmse: 0.0193554\n",
      "[18000]\tvalid_0's rmse: 0.0193402\n",
      "[18200]\tvalid_0's rmse: 0.0193306\n",
      "[18400]\tvalid_0's rmse: 0.0193191\n",
      "[18600]\tvalid_0's rmse: 0.0193044\n",
      "[18800]\tvalid_0's rmse: 0.0192909\n",
      "[19000]\tvalid_0's rmse: 0.0192759\n",
      "[19200]\tvalid_0's rmse: 0.0192633\n",
      "[19400]\tvalid_0's rmse: 0.0192511\n",
      "[19600]\tvalid_0's rmse: 0.0192399\n",
      "[19800]\tvalid_0's rmse: 0.0192284\n",
      "[20000]\tvalid_0's rmse: 0.0192197\n",
      "[20200]\tvalid_0's rmse: 0.0192092\n",
      "[20400]\tvalid_0's rmse: 0.0192008\n",
      "[20600]\tvalid_0's rmse: 0.0191945\n",
      "[20800]\tvalid_0's rmse: 0.0191871\n",
      "[21000]\tvalid_0's rmse: 0.0191766\n",
      "[21200]\tvalid_0's rmse: 0.0191692\n",
      "[21400]\tvalid_0's rmse: 0.019163\n",
      "[21600]\tvalid_0's rmse: 0.0191561\n",
      "[21800]\tvalid_0's rmse: 0.0191504\n",
      "[22000]\tvalid_0's rmse: 0.0191451\n",
      "[22200]\tvalid_0's rmse: 0.0191366\n",
      "[22400]\tvalid_0's rmse: 0.0191321\n",
      "[22600]\tvalid_0's rmse: 0.0191246\n",
      "[22800]\tvalid_0's rmse: 0.0191171\n",
      "[23000]\tvalid_0's rmse: 0.0191103\n",
      "[23200]\tvalid_0's rmse: 0.0191057\n",
      "[23400]\tvalid_0's rmse: 0.0191003\n",
      "[23600]\tvalid_0's rmse: 0.0190948\n",
      "[23800]\tvalid_0's rmse: 0.0190913\n",
      "[24000]\tvalid_0's rmse: 0.0190843\n",
      "[24200]\tvalid_0's rmse: 0.0190812\n",
      "[24400]\tvalid_0's rmse: 0.0190716\n",
      "[24600]\tvalid_0's rmse: 0.019063\n",
      "[24800]\tvalid_0's rmse: 0.0190558\n",
      "[25000]\tvalid_0's rmse: 0.01905\n",
      "[25200]\tvalid_0's rmse: 0.0190474\n",
      "[25400]\tvalid_0's rmse: 0.0190453\n",
      "[25600]\tvalid_0's rmse: 0.0190413\n",
      "[25800]\tvalid_0's rmse: 0.0190363\n",
      "[26000]\tvalid_0's rmse: 0.019032\n",
      "[26200]\tvalid_0's rmse: 0.0190277\n",
      "[26400]\tvalid_0's rmse: 0.0190228\n",
      "[26600]\tvalid_0's rmse: 0.0190162\n",
      "[26800]\tvalid_0's rmse: 0.0190149\n",
      "[27000]\tvalid_0's rmse: 0.0190117\n",
      "[27200]\tvalid_0's rmse: 0.0190076\n",
      "[27400]\tvalid_0's rmse: 0.0190029\n",
      "[27600]\tvalid_0's rmse: 0.0190045\n",
      "[27800]\tvalid_0's rmse: 0.0190012\n",
      "[28000]\tvalid_0's rmse: 0.0189986\n",
      "[28200]\tvalid_0's rmse: 0.0189953\n",
      "[28400]\tvalid_0's rmse: 0.0189954\n",
      "[28600]\tvalid_0's rmse: 0.0189909\n",
      "[28800]\tvalid_0's rmse: 0.0189895\n",
      "[29000]\tvalid_0's rmse: 0.0189875\n",
      "[29200]\tvalid_0's rmse: 0.0189872\n",
      "[29400]\tvalid_0's rmse: 0.0189831\n",
      "[29600]\tvalid_0's rmse: 0.0189804\n",
      "[29800]\tvalid_0's rmse: 0.0189803\n",
      "[30000]\tvalid_0's rmse: 0.0189815\n",
      "[30200]\tvalid_0's rmse: 0.0189796\n",
      "Early stopping, best iteration is:\n",
      "[29844]\tvalid_0's rmse: 0.0189792\n",
      "Test RMSE :  0.12588161367859751\n",
      "CV RMSE :  0.13979503719615177\n"
     ]
    }
   ],
   "source": [
    "IterationNum = 1\n",
    "for train_index, test_index in kf.split(train02):\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    MOD_DATA_2_TRAIN, MOD_DATA_2_TEST = train02.iloc[train_index], train02.iloc[test_index]\n",
    "    \n",
    "    parameters = {  'objective': 'regression',\n",
    "                    'metric': 'rmse',\n",
    "                    'boosting': 'gbdt',\n",
    "                    'feature_fraction': 0.2,\n",
    "                    'bagging_fraction': 0.9,\n",
    "                    'bagging_freq' : 0,\n",
    "                    'learning_rate': 0.005,\n",
    "                    'min_data_in_leaf': 2,\n",
    "                    'max_depth': 4,\n",
    "                    'seed': 500,\n",
    "                    'max_bin': 75,\n",
    "                    'min_data_in_bin': 5,\n",
    "                    'verbosity': -1,\n",
    "                    'silent': -1  }\n",
    "    \n",
    "    X_TRAIN = pd.DataFrame(MOD_DATA_2_TRAIN[feature_names])\n",
    "    Y_TRAIN = MOD_DATA_2_TRAIN[\"Price\"]\n",
    "                \n",
    "    X_TEST = pd.DataFrame(MOD_DATA_2_TEST[feature_names])\n",
    "    Y_TEST = MOD_DATA_2_TEST[\"Price\"]\n",
    "    \n",
    "    train_data = lgb.Dataset(X_TRAIN,\n",
    "                             label = Y_TRAIN)\n",
    "    valid_data = lgb.Dataset(X_TEST,\n",
    "                             label = Y_TEST)\n",
    "    lgb_model = lgb.train(parameters,\n",
    "                          train_data,\n",
    "                          valid_sets = valid_data,\n",
    "                          num_boost_round = 10000000,\n",
    "                          early_stopping_rounds = 400,\n",
    "                          verbose_eval = 200)\n",
    "    MOD_DATA_2_TEST['Predicted_Model_Value'] = lgb_model.predict(pd.DataFrame(MOD_DATA_2_TEST[feature_names]))\n",
    "    \n",
    "    MOD_DATA_2_TEST[\"Price\"] = np.log((MOD_DATA_2_TEST[\"Price\"]**10)+1)\n",
    "    MOD_DATA_2_TEST['Predicted_Model_Value'] = np.log((MOD_DATA_2_TEST['Predicted_Model_Value']**10)+1)\n",
    "    \n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = MOD_DATA_2_TEST.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        test_scored = lgb_model.predict(pd.DataFrame(test01[feature_names]))\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,MOD_DATA_2_TEST])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        test_scored = test_scored + lgb_model.predict(pd.DataFrame(test01[feature_names]))\n",
    "                    \n",
    "    IterationNum = IterationNum + 1\n",
    "    \n",
    "    print(\"Test RMSE : \",sqrt(mean_squared_error(MOD_DATA_2_TEST[\"Price\"], MOD_DATA_2_TEST['Predicted_Model_Value'])))\n",
    "    \n",
    "print(\"CV RMSE : \",sqrt(mean_squared_error(CV_SCORED_DATA[\"Price\"], CV_SCORED_DATA['Predicted_Model_Value'])))\n",
    "#CV RMSE :  0.14204806806938736 0.13979503719615177\n",
    "#LB: 0.9463"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=1234, minmax=(0.9229671020236232, 1.5432109503595928), mean=1.2001989522212881, variance=0.0104433286774734, skewness=0.6665188295432515, kurtosis=0.18619406186459653)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "test_scored2 = test_scored / 5.0\n",
    "stats.describe(test_scored2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_scored3 = pd.DataFrame({'Price' : (test_scored2**10)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.892957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.032972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17.113367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.261667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.619814</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Price\n",
       "0   2.892957\n",
       "1   3.032972\n",
       "2  17.113367\n",
       "3   4.261667\n",
       "4   4.619814"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_scored3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV_SCORED_DATA.to_csv(\"C:\\\\Kaggle\\\\Cars\\\\CV_Scored\\\\20190716_LGB01copy_CVTRAIN_DS.csv\",\n",
    "                      index = False)\n",
    "test_scored3.to_csv(\"C:\\\\Kaggle\\\\Cars\\\\Submission\\\\20190716_LGB01copy_TEST_DS.csv\",\n",
    "                    index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
